package craw;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;

import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.List;

/**
 * User: zigang.wang
 * Date: 12-9-12
 * Time: 下午2:47
 */
public class Get233 {

    static DefaultHttpClient httpClient = new DefaultHttpClient();

    public static String get(String url, String encoding) throws Exception {
//        System.out.println(url);
        url = StringUtils.replace(url, " ", "%20");
        HttpGet httpGet = new HttpGet(url);
        HttpResponse res = httpClient.execute(httpGet);
        return getContent(res, encoding);
    }


    public static String getContent(HttpResponse res, String encoding) throws Exception {
        HttpEntity ent = res.getEntity();
        String result = IOUtils.toString(ent.getContent(), encoding);
        EntityUtils.consume(ent);
        return result;
    }


    public static List<String> getArticaList(String url, int size) throws Exception {
        String content = get(url, "gb2312");
        List<String> list = getArticles(content);
        for (int i = 2; i <= size; i++) {
            String pages = get(url + "index0" + i + ".html", "gb2312");
            List<String> pageArticle = getArticles(pages);
            if (pageArticle != null && pageArticle.size() > 0) {
                list.addAll(pageArticle);
            }
        }
        return list;
    }

    public static String getNeedContent(String url) throws Exception {
        if (!StringUtils.contains(url, "http://")) {
            url = "http://www.233.com/" + url;
        }
        String content = get(url, "gb2312");
        String needContent = RegexUtil.getValue(content, "<div id=\"N_content\" class=\"mar10\">(.*?)<font color=\"#f5fdff\">");
        String pageContent = RegexUtil.getValue(content, "分页开始 -->(.*?)分页结束 -->");
        List<String> pageNum = RegexUtil.getValues(pageContent, "<a class=page href=\"(.*?)\" target=_self>");
        for (String s : pageNum) {
            Thread.sleep(1000);
            String pageUrl = StringUtils.substringBeforeLast(url, "/") + "/" + s;
            String needs = RegexUtil.getValue(get(pageUrl, "gb2312"), "<div id=\"N_content\" class=\"mar10\">(.*?)<font color=\"#f5fdff\">");
            needContent += needs;
        }
        return needContent;
    }

    private static List<String> getArticles(String pages) {
        return RegexUtil.getValues(pages, "<A target=\"_blank\" class=lst14 href=\"(.*?)\" title=\"");
    }

    public static void saveContent(String content, String fileName) throws IOException {
        if(new File(fileName).exists()){
            return;
        }
        System.out.println("write to " + fileName);
        FileUtils.writeStringToFile(new File(fileName), content);
        String path = StringUtils.substringBeforeLast(fileName, ".");
        final String picPath = path + "-img";
        List<String> imgs = RegexUtil.getValues(content, "src=\"(.*?)\"");
        for (String s : imgs) {
            if (StringUtils.containsIgnoreCase("jpg,jpeg,png,gif", StringUtils.substringAfterLast(s, "."))) {
                System.out.println("get img:" + s);
                String picName = StringUtils.substringAfterLast(s, "/");
                if (!StringUtils.contains(s, "http://")) {
                    s = "http://www.233.com/" + s;
                }
                FileUtils.copyURLToFile(new URL(s), new File(picPath + "/" + picName));
            }else{
                System.out.println("ingnore:" + s);
            }
        }

    }

    public static void main(String[] args) throws Exception {
        List<String> articles = getArticaList("http://www.233.com/gct/logic/moniti/", 8);
        System.out.println(articles.size());
        String fileRoot = "E:\\233\\logic\\";
        for (int i = 0, articlesSize = articles.size(); i < articlesSize; i++) {
            try {
                String s = articles.get(i);
                Thread.sleep(3000);
                System.out.println(i + " get:" + s);
                String content = getNeedContent(s);
                String fileName = StringUtils.substringAfterLast(s, "/");
                String path = StringUtils.substringBefore(fileName, ".");
                saveContent(content, fileRoot + path + "\\" + path + ".txt");
            } catch (Exception e) {
                Thread.sleep(3000);
                e.printStackTrace();
            }
        }
    }
}
